# plotly standard imports
import plotly.graph_objs as go
import chart_studio.plotly as py

# Cufflinks wrapper on plotly
import cufflinks

# Data science imports
import pandas as pd
import numpy as np

# Options for pandas
pd.options.display.max_columns = 30

# Display all cell outputs
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

from plotly.offline import iplot, init_notebook_mode
cufflinks.go_offline(connected=True)
init_notebook_mode(connected=True)

# Set global theme
cufflinks.set_config_file(world_readable=True, theme='pearl')

import os
import mlflow

trackng_url = os.getenv('MLFLOW_TRACKING_URI')
print(trackng_url)
mlflow.set_tracking_uri(trackng_url)

http://mlflow:5000

Prepare dataset

from src.prepare_datasets import get_prepared_datasets

# five minute dataset
train, test = get_prepared_datasets()

train

	timestamp	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
0	2016-08-10 15:53:00	0.000158	0.000156	0.000158	0.000156	0.000149	0.000773	0.000717	0.000860	0.557937	1.000000	0.0	0.000791	0.000777	0.000744	0.500000	0.500000	0.000000
1	2016-08-10 15:59:00	0.000417	0.000415	0.000375	0.000415	0.041003	0.001443	0.002706	0.000186	0.560160	1.000000	1.0	0.001459	0.001447	0.001417	1.000000	1.000000	0.000000
2	2016-08-10 16:04:00	0.000495	0.000495	0.000490	0.000495	0.026081	0.001804	0.003219	0.000392	0.562563	1.000000	1.0	0.001667	0.001655	0.001626	0.998519	0.999259	0.000000
3	2016-08-10 16:09:00	0.000495	0.000495	0.000492	0.000495	0.050928	0.001985	0.003343	0.000631	0.564397	1.000000	1.0	0.001667	0.001655	0.001626	1.000000	0.999506	0.000000
4	2016-08-10 16:14:00	0.000470	0.000468	0.000463	0.000463	0.001854	0.002060	0.003300	0.000824	0.565480	0.893887	1.0	0.001667	0.001655	0.001626	0.903704	0.967407	0.000000
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
363254	2020-04-22 12:16:00	0.064202	0.064200	0.064156	0.064156	0.000249	0.332335	0.330019	0.333521	0.558528	0.420486	0.0	0.331758	0.332473	0.334215	0.003448	0.336163	0.018243
363255	2020-04-22 12:21:00	0.064201	0.064199	0.064176	0.064174	0.000145	0.332314	0.330029	0.333469	0.557445	0.432412	0.0	0.331758	0.332473	0.334215	0.065517	0.071695	0.017497
363256	2020-04-22 12:26:00	0.064178	0.064177	0.064168	0.064177	0.000190	0.332315	0.330027	0.333474	0.556628	0.434500	0.0	0.331758	0.332473	0.334215	0.075862	0.048276	0.016023
363257	2020-04-22 12:31:00	0.064219	0.064228	0.064218	0.064223	0.000019	0.332328	0.330017	0.333510	0.556401	0.466894	0.0	0.331758	0.332473	0.334215	0.234483	0.125287	0.015699
363258	2020-04-22 12:36:00	0.064184	0.064183	0.064184	0.064182	0.000004	0.332343	0.329990	0.333567	0.555885	0.442321	0.0	0.331758	0.332473	0.334215	0.093103	0.134483	0.015156

363259 rows × 18 columns

train.index = pd.to_datetime(train.pop('timestamp'))
test.index = pd.to_datetime(test.pop('timestamp'))

train[::15].iplot(subplots=True)

test[::15].iplot(subplots=True)

len(test)
prediction_size = 3*31*24*15 # last three monthes

# for later prediction test
unseen_data = test[-prediction_size:]
test_only = test[0:-prediction_size]

len(test_only)

unseen_data

90815

57335

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-11-13 07:34:00	0.157233	0.157513	0.157233	0.157322	0.000473	0.817511	0.813085	0.819089	0.532898	0.302029	0.0	0.814454	0.816520	0.824145	0.065032	0.054866	0.054889
2020-11-13 07:39:00	0.157518	0.157553	0.157518	0.157544	0.000427	0.817468	0.813163	0.818923	0.531211	0.364549	0.0	0.814454	0.816520	0.824145	0.225095	0.096709	0.055164
2020-11-13 07:44:00	0.157756	0.157764	0.157693	0.157764	0.000022	0.817390	0.813169	0.818761	0.532080	0.419757	0.0	0.814046	0.816316	0.824145	0.382968	0.224365	0.055124
2020-11-13 07:49:00	0.157806	0.157814	0.157766	0.157773	0.000376	0.817312	0.813163	0.818610	0.533132	0.421938	0.0	0.813950	0.816268	0.824145	0.402098	0.336720	0.050865
2020-11-13 07:54:00	0.157846	0.157844	0.157836	0.157834	0.000290	0.817260	0.813151	0.818517	0.534766	0.437745	0.0	0.813224	0.815904	0.824145	0.447761	0.410942	0.047562
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2021-03-09 18:30:00	0.537239	0.537323	0.537239	0.537323	0.000012	2.781359	2.767361	2.785575	0.479207	0.458981	1.0	2.769829	2.780198	2.807013	0.566187	0.511059	0.248703
2021-03-09 18:35:00	0.537776	0.537956	0.537676	0.537856	0.000266	2.780858	2.766114	2.785832	0.488875	0.483544	1.0	2.769829	2.780198	2.807013	0.723634	0.527857	0.239622
2021-03-09 18:40:00	0.536006	0.536056	0.536006	0.536005	0.000056	2.780218	2.766060	2.784600	0.481352	0.412709	0.0	2.769829	2.780198	2.807013	0.236027	0.508616	0.261776
2021-03-09 18:45:00	0.536539	0.536538	0.536539	0.536538	0.000006	2.779332	2.764294	2.784610	0.480906	0.438007	0.0	2.771712	2.781142	2.806270	0.408327	0.455996	0.248882
2021-03-09 18:50:00	0.534629	0.534648	0.534629	0.534648	0.000167	2.777951	2.763188	2.782956	0.465204	0.375468	0.0	2.768158	2.778523	2.804076	0.005376	0.216577	0.271614

33480 rows × 17 columns

Start training

from pycaret.regression import *

exp_reg_btc = setup(
    data = train, test_data = test_only, target = 'close', 
    session_id = 123, 
    log_experiment = True, log_plots=True, log_profile=True, experiment_name = 'bitcoin_close_5m_norm',
    use_gpu = True,
    data_split_shuffle = False,
    feature_selection = True, feature_selection_threshold = 0.6
)

	Description	Value
0	session_id	123
1	Target	close
2	Original Data	(363259, 17)
3	Missing Values	False
4	Numeric Features	15
5	Categorical Features	1
6	Ordinal Features	False
7	High Cardinality Features	False
8	High Cardinality Method	None
9	Transformed Train Set	(363259, 6)
10	Transformed Test Set	(57335, 6)
11	Shuffle Train-Test	False
12	Stratify Train-Test	False
13	Fold Generator	KFold
14	Fold Number	10
15	CPU Jobs	-1
16	Use GPU	True
17	Log Experiment	True
18	Experiment Name	bitcoin_close_5m_norm
19	USI	1e93
20	Imputation Type	simple
21	Iterative Imputation Iteration	None
22	Numeric Imputer	mean
23	Iterative Imputation Numeric Model	None
24	Categorical Imputer	constant
25	Iterative Imputation Categorical Model	None
26	Unknown Categoricals Handling	least_frequent
27	Normalize	False
28	Normalize Method	None
29	Transformation	False
30	Transformation Method	None
31	PCA	False
32	PCA Method	None
33	PCA Components	None
34	Ignore Low Variance	False
35	Combine Rare Levels	False
36	Rare Level Threshold	None
37	Numeric Binning	False
38	Remove Outliers	False
39	Outliers Threshold	None
40	Remove Multicollinearity	False
41	Multicollinearity Threshold	None
42	Clustering	False
43	Clustering Iteration	None
44	Polynomial Features	False
45	Polynomial Degree	None
46	Trignometry Features	False
47	Polynomial Threshold	None
48	Group Features	False
49	Feature Selection	True
50	Feature Selection Method	classic
51	Features Selection Threshold	0.600000
52	Feature Interaction	False
53	Feature Ratio	False
54	Interaction Threshold	None
55	Transform Target	False
56	Transform Target Method	box-cox

TODO

• try use all features from add_ta_features and set feature_selection=True
• try add featues by tsfreash
• try extract features by tsfreash
• try remove trend by sktime

best = compare_models(n_select=3)

	Model	MAE	MSE	RMSE	R2	RMSLE	MAPE	TT (Sec)
lightgbm	Light Gradient Boosting Machine	0.0212	0.0007	0.0251	-36.0334	0.0237	5.7096	0.5070
xgboost	Extreme Gradient Boosting	0.0212	0.0007	0.0252	-36.1482	0.0238	5.7947	1.4410
gbr	Gradient Boosting Regressor	0.0215	0.0007	0.0254	-37.2733	0.0240	5.7886	53.1190
et	Extra Trees Regressor	0.0216	0.0007	0.0257	-38.2130	0.0243	5.9779	37.6070
rf	Random Forest Regressor	0.0216	0.0007	0.0257	-39.0340	0.0243	6.0995	26.9150
catboost	CatBoost Regressor	0.0232	0.0008	0.0275	-41.0420	0.0259	5.8071	4.1250
knn	K Neighbors Regressor	0.0240	0.0008	0.0285	-47.5957	0.0269	5.9866	22.1830
ada	AdaBoost Regressor	0.0228	0.0008	0.0264	-50.5735	0.0249	6.9411	11.6370
dt	Decision Tree Regressor	0.0270	0.0011	0.0332	-59.0661	0.0313	6.2116	2.9450
huber	Huber Regressor	0.0277	0.0013	0.0330	-90.1986	0.0302	9.5374	2.2730
lr	Linear Regression	0.0265	0.0011	0.0304	-94.0724	0.0283	9.7695	0.0500
br	Bayesian Ridge	0.0265	0.0011	0.0304	-94.0731	0.0283	9.7696	0.0910
lar	Least Angle Regression	0.0265	0.0011	0.0304	-94.0792	0.0283	9.7700	0.0300
ridge	Ridge Regression	0.0265	0.0011	0.0304	-94.1452	0.0283	9.7721	0.0280
omp	Orthogonal Matching Pursuit	0.0270	0.0011	0.0307	-96.4394	0.0287	9.7132	0.0290
lasso	Lasso Regression	0.0320	0.0015	0.0348	-156.3319	0.0329	12.3179	0.0600
en	Elastic Net	0.0320	0.0015	0.0348	-156.3319	0.0329	12.3179	0.0530
llar	Lasso Least Angle Regression	0.0320	0.0015	0.0348	-156.3319	0.0329	12.3179	0.0320
par	Passive Aggressive Regressor	0.0470	0.0032	0.0492	-397.5062	0.0463	18.5913	0.2410

best

[LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
               importance_type='split', learning_rate=0.1, max_depth=-1,
               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,
               random_state=123, reg_alpha=0.0, reg_lambda=0.0, silent=True,
               subsample=1.0, subsample_for_bin=200000, subsample_freq=0),
 XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=0,
              importance_type='gain', interaction_constraints='',
              learning_rate=0.300000012, max_delta_step=0, max_depth=6,
              min_child_weight=1, missing=nan, monotone_constraints='()',
              n_estimators=100, n_jobs=-1, num_parallel_tree=1,
              objective='reg:squarederror', random_state=123, reg_alpha=0,
              reg_lambda=1, scale_pos_weight=1, subsample=1,
              tree_method='gpu_hist', validate_parameters=1, verbosity=0),
 GradientBoostingRegressor(alpha=0.9, ccp_alpha=0.0, criterion='friedman_mse',
                           init=None, learning_rate=0.1, loss='ls', max_depth=3,
                           max_features=None, max_leaf_nodes=None,
                           min_impurity_decrease=0.0, min_impurity_split=None,
                           min_samples_leaf=1, min_samples_split=2,
                           min_weight_fraction_leaf=0.0, n_estimators=100,
                           n_iter_no_change=None, presort='deprecated',
                           random_state=123, subsample=1.0, tol=0.0001,
                           validation_fraction=0.1, verbose=0, warm_start=False)]

models()

	Name	Reference	Turbo
ID
lr	Linear Regression	sklearn.linear_model._base.LinearRegression	True
lasso	Lasso Regression	sklearn.linear_model._coordinate_descent.Lasso	True
ridge	Ridge Regression	sklearn.linear_model._ridge.Ridge	True
en	Elastic Net	sklearn.linear_model._coordinate_descent.Elast...	True
lar	Least Angle Regression	sklearn.linear_model._least_angle.Lars	True
llar	Lasso Least Angle Regression	sklearn.linear_model._least_angle.LassoLars	True
omp	Orthogonal Matching Pursuit	sklearn.linear_model._omp.OrthogonalMatchingPu...	True
br	Bayesian Ridge	sklearn.linear_model._bayes.BayesianRidge	True
ard	Automatic Relevance Determination	sklearn.linear_model._bayes.ARDRegression	False
par	Passive Aggressive Regressor	sklearn.linear_model._passive_aggressive.Passi...	True
ransac	Random Sample Consensus	sklearn.linear_model._ransac.RANSACRegressor	False
tr	TheilSen Regressor	sklearn.linear_model._theil_sen.TheilSenRegressor	False
huber	Huber Regressor	sklearn.linear_model._huber.HuberRegressor	True
kr	Kernel Ridge	sklearn.kernel_ridge.KernelRidge	False
svm	Support Vector Regression	sklearn.svm._classes.SVR	False
knn	K Neighbors Regressor	sklearn.neighbors._regression.KNeighborsRegressor	True
dt	Decision Tree Regressor	sklearn.tree._classes.DecisionTreeRegressor	True
rf	Random Forest Regressor	sklearn.ensemble._forest.RandomForestRegressor	True
et	Extra Trees Regressor	sklearn.ensemble._forest.ExtraTreesRegressor	True
ada	AdaBoost Regressor	sklearn.ensemble._weight_boosting.AdaBoostRegr...	True
gbr	Gradient Boosting Regressor	sklearn.ensemble._gb.GradientBoostingRegressor	True
mlp	MLP Regressor	sklearn.neural_network._multilayer_perceptron....	False
xgboost	Extreme Gradient Boosting	xgboost.sklearn.XGBRegressor	True
lightgbm	Light Gradient Boosting Machine	lightgbm.sklearn.LGBMRegressor	True
catboost	CatBoost Regressor	catboost.core.CatBoostRegressor	True

plot_model(best[0], plot = 'learning')

plot_model(best[1], plot = 'learning')

plot_model(best[0], plot = 'error')

plot_model(best[0])

lgbmr = tune_model(best[0], choose_better = True)

	MAE	MSE	RMSE	R2	RMSLE	MAPE
0	0.0269	0.0008	0.0282	-307.3366	0.0277	53.7458
1	0.0297	0.0010	0.0322	-27.3686	0.0312	3.1158
2	0.0240	0.0007	0.0272	-3.4374	0.0260	0.8573
3	0.0348	0.0019	0.0434	-0.9534	0.0391	0.2839
4	0.0149	0.0003	0.0181	-1.9856	0.0170	0.2073
5	0.0183	0.0005	0.0228	-10.5516	0.0218	0.3074
6	0.0173	0.0004	0.0205	-54.3538	0.0196	0.5243
7	0.0154	0.0004	0.0191	0.3232	0.0177	0.1948
8	0.0228	0.0007	0.0262	-3.6407	0.0245	0.2650
9	0.0190	0.0006	0.0235	-2.0820	0.0219	0.2690
Mean	0.0223	0.0007	0.0261	-41.1386	0.0246	5.9770
SD	0.0062	0.0004	0.0071	90.2114	0.0064	15.9452

lgbmr

LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
              importance_type='split', learning_rate=0.1, max_depth=-1,
              min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,
              n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,
              random_state=123, reg_alpha=0.0, reg_lambda=0.0, silent=True,
              subsample=1.0, subsample_for_bin=200000, subsample_freq=0)

plot_model(lgbmr, plot='feature')

evaluate_model(lgbmr)

predict_model(lgbmr)

	Model	MAE	MSE	RMSE	R2	RMSLE	MAPE
0	Light Gradient Boosting Machine	0.0296	0.0011	0.0337	-2.9392	0.0309	0.2859

	momentum_rsi	volatility_kchi_0.0	momentum_stoch_signal	volume	trend_macd	volatility_atr	close	Label
0	0.504099	1.0	0.244828	0.000007	0.556291	0.015945	0.064273	0.071777
1	0.508609	0.0	0.310345	0.000003	0.556688	0.014651	0.064280	0.074599
2	0.523927	0.0	0.449425	0.000017	0.557212	0.013762	0.064303	0.066733
3	0.546732	0.0	0.523526	0.000260	0.557928	0.013250	0.064338	0.049312
4	0.570471	0.0	0.632029	0.000005	0.558808	0.012991	0.064375	0.070173
...	...	...	...	...	...	...	...	...
57330	0.449912	1.0	0.662669	0.000027	0.548167	0.048742	0.158219	0.088362
57331	0.404800	1.0	0.460459	0.000135	0.545853	0.050947	0.157975	0.084884
57332	0.381000	1.0	0.293049	0.000154	0.542969	0.052368	0.157834	0.085809
57333	0.375895	1.0	0.153386	0.000073	0.540621	0.049137	0.157804	0.087726
57334	0.353756	1.0	0.077440	0.001309	0.537857	0.048734	0.157673	0.072141

57335 rows × 8 columns

final_lgbmr = finalize_model(lgbmr)

Finished loading model, total used 100 iterations
Finished loading model, total used 100 iterations
Finished loading model, total used 100 iterations

save_model(final_lgbmr, './saved_models/Final_LGBMR_5m_bitcoin_norm')

Transformation Pipeline and Model Succesfully Saved

(Pipeline(memory=None,
          steps=[('dtypes',
                  DataTypes_Auto_infer(categorical_features=[],
                                       display_types=True, features_todrop=[],
                                       id_columns=[], ml_usecase='regression',
                                       numerical_features=[], target='close',
                                       time_features=[])),
                 ('imputer',
                  Simple_Imputer(categorical_strategy='not_available',
                                 fill_value_categorical=None,
                                 fill_value_numerical=None,
                                 numeric_strategy='...
                  LGBMRegressor(boosting_type='gbdt', class_weight=None,
                                colsample_bytree=1.0, importance_type='split',
                                learning_rate=0.1, max_depth=-1,
                                min_child_samples=20, min_child_weight=0.001,
                                min_split_gain=0.0, n_estimators=100, n_jobs=-1,
                                num_leaves=31, objective=None, random_state=123,
                                reg_alpha=0.0, reg_lambda=0.0, silent=True,
                                subsample=1.0, subsample_for_bin=200000,
                                subsample_freq=0)]],
          verbose=False),
 './saved_models/Final_LGBMR_5m_bitcoin_norm.pkl')

unseen_predictions = predict_model(final_lgbmr, data=unseen_data)
unseen_predictions.head()

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr	Label
timestamp
2020-11-13 07:34:00	0.157233	0.157513	0.157233	0.157322	0.000473	0.817511	0.813085	0.819089	0.532898	0.302029	0.0	0.814454	0.816520	0.824145	0.065032	0.054866	0.054889	0.084302
2020-11-13 07:39:00	0.157518	0.157553	0.157518	0.157544	0.000427	0.817468	0.813163	0.818923	0.531211	0.364549	0.0	0.814454	0.816520	0.824145	0.225095	0.096709	0.055164	0.083622
2020-11-13 07:44:00	0.157756	0.157764	0.157693	0.157764	0.000022	0.817390	0.813169	0.818761	0.532080	0.419757	0.0	0.814046	0.816316	0.824145	0.382968	0.224365	0.055124	0.086180
2020-11-13 07:49:00	0.157806	0.157814	0.157766	0.157773	0.000376	0.817312	0.813163	0.818610	0.533132	0.421938	0.0	0.813950	0.816268	0.824145	0.402098	0.336720	0.050865	0.078496
2020-11-13 07:54:00	0.157846	0.157844	0.157836	0.157834	0.000290	0.817260	0.813151	0.818517	0.534766	0.437745	0.0	0.813224	0.815904	0.824145	0.447761	0.410942	0.047562	0.078341

from pycaret.utils import check_metric
check_metric(unseen_predictions['close'], unseen_predictions['Label'], 'R2')

-2.5982

unseen_predictions[['close', 'Label']].iplot()